# usage: parse-news.py news_text out_file

import os, sys, subprocess

mystem_path = "mystem -n -e cp1251"
outTmpFileName = "news-out.tmp"
outTmpFileName1 = "news-out1.tmp"

def Normalize(text):
	news = {}
	currentMarker = ""
	content = ""
	bNewNews = True
	
	words = []

	with open(text, encoding = "cp1251") as textFile:
		for line in textFile:
			if (line.startswith("/news")):
				bNewNews = True
				currentMarker = line.replace("\n", "")
				continue
			if (bNewNews):
				bNewNews = False
				news[currentMarker] = content
				content = ""
				continue
			content += line.replace("\n", "")
		news[currentMarker] = content

	i = 0
	writeFile = open(sys.argv[2], 'w', encoding = "cp1251")
	for newsItem in news:
		with open (outTmpFileName, "w", encoding = "cp1251") as outTmpNewsFile:
			outTmpNewsFile.write(news[newsItem])
		subprocess.check_output("%s %s %s" % (mystem_path, outTmpFileName, outTmpFileName1), shell = True)
		with open(outTmpFileName1, encoding = "cp1251") as outputFile:
			for line in outputFile:
				wordform = line[:line.find("{")]
				#words.append((wordform, line[len(wordform) + 1:-2].split("|")))
				writeFile.write("%s: %s\n" % (wordform, line[len(wordform) + 1:-2]))
			writeFile.write("\n\n")
			print("sentence %d parsed" % i)
			i += 1

#    for word in words:
#        print (word)
#    print(words)
	os.remove(outTmpFileName)

	return words

if (len(sys.argv) < 3):
	print ("usage: parse-news.py news_text out_file")
	sys.exit(1)

Normalize(sys.argv[1])